package com.study.bigdata.spark.sql

import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}

object Spark01_SparkSQL_Basic {
  def main(args: Array[String]): Unit = {
    // TODO 创建SparkSql的运行环境   环境+对象+隐式转换
    val sparkConf = new SparkConf().setMaster("local[*]").setAppName("sparkSQL")
    val spark = SparkSession.builder().config(sparkConf).getOrCreate()
    import spark.implicits._
    // TODO 执行逻辑

    //DataFrame
//    val df: DataFrame = spark.read.json("data/user.json")
//    df.show()

    //DataFrame ==>SQL
//    df.createOrReplaceTempView("user")
//    spark.sql("select avg(age) from user").show

    //DataFrame ==>DSL
    //在使用DataFrame时，如果设计转换操作，需要引入转换规则
//    df.select("age","username").show
//    df.select($"age"+1 ).show
//    df.select('age+1 ).show

    //DataSet
    //DataFrame其实是特定泛型的DataSet
    //type DataFrame = Dataset[Row]
//    val seq = Seq(1, 2, 3, 4)
//    val ds = seq.toDS()
//    ds.show()

    //RDD<=>DataFrame
    //RDD只有数据
    val rdd = spark.sparkContext.makeRDD(List(
      (1, "zhangsan", 20),
      (2, "lisi", 20)
    ))
    //DataFrame数据+结构
    val df: DataFrame = rdd.toDF("id", "name", "age")
    val rowRDD: RDD[Row] = df.rdd

    //DataFrame<=>DataSet
    //通过样例类就能把只有数据和结构的DataFrame转换为既有数据又有结构又有类型的DataSet
    val ds: Dataset[User] = df.as[User]
    val df1: DataFrame = ds.toDF()

    //RDD<=>DataSet
    //也是通过样例类实现转换
    val ds1: Dataset[User] = rdd.map {
      case (id, name, age) => {
        User(id, name, age)
      }
    }.toDS()
    val userRDD: RDD[User] = ds1.rdd

    // TODO 关闭环境
    spark.close()
  }
  case class User(id:Int,name:String,age:Int)
}
